{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# dropout随机失活降低过拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "下面这块是不用随机失活就有的效果："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From <ipython-input-2-8b1e9efd65e5>:36: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "[[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]] [[ 0.  0.  0. ...,  0.  0.  0.]\n",
      " [ 0.  0.  0. ...,  1.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]\n",
      " ..., \n",
      " [ 0.  0.  0. ...,  1.  0.  0.]\n",
      " [ 0.  0.  0. ...,  0.  1.  0.]\n",
      " [ 0.  0.  0. ...,  0.  0.  0.]]\n"
     ]
    },
    {
     "ename": "InternalError",
     "evalue": "Blas GEMM launch failed : a.shape=(200, 784), b.shape=(784, 2000), m=200, n=2000, k=784\n\t [[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](_arg_Placeholder_0_0/_5, Variable/read)]]\n\t [[Node: add_7/_7 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_80_add_7\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\n\nCaused by op 'MatMul', defined at:\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tornado\\ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2698, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2802, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2862, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-2-8b1e9efd65e5>\", line 17, in <module>\n    L1=tf.nn.tanh(tf.matmul(x,W1)+b1)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py\", line 2022, in matmul\n    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\ops\\gen_math_ops.py\", line 2799, in _mat_mul\n    name=name)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3160, in create_op\n    op_def=op_def)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1625, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInternalError (see above for traceback): Blas GEMM launch failed : a.shape=(200, 784), b.shape=(784, 2000), m=200, n=2000, k=784\n\t [[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](_arg_Placeholder_0_0/_5, Variable/read)]]\n\t [[Node: add_7/_7 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_80_add_7\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mInternalError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1349\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1350\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1351\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1328\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1329\u001b[1;33m                                    status, run_metadata)\n\u001b[0m\u001b[0;32m   1330\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[1;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[0;32m    472\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 473\u001b[1;33m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[0;32m    474\u001b[0m     \u001b[1;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInternalError\u001b[0m: Blas GEMM launch failed : a.shape=(200, 784), b.shape=(784, 2000), m=200, n=2000, k=784\n\t [[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](_arg_Placeholder_0_0/_5, Variable/read)]]\n\t [[Node: add_7/_7 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_80_add_7\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[1;31mInternalError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-2-8b1e9efd65e5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     50\u001b[0m             \u001b[0mbatch_xs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_ys\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmnist\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnext_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     51\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_xs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mbatch_ys\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 52\u001b[1;33m             \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_step\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mbatch_xs\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mbatch_ys\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mkeep_prob\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m1.0\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     54\u001b[0m         \u001b[0mtest_acc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maccuracy\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mmnist\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimages\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mmnist\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mkeep_prob\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m1.0\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    893\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 895\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    896\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1126\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1127\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1128\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1129\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1130\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1342\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1343\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1344\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1345\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1346\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1361\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1362\u001b[0m           \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1363\u001b[1;33m       \u001b[1;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1364\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1365\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mInternalError\u001b[0m: Blas GEMM launch failed : a.shape=(200, 784), b.shape=(784, 2000), m=200, n=2000, k=784\n\t [[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](_arg_Placeholder_0_0/_5, Variable/read)]]\n\t [[Node: add_7/_7 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_80_add_7\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\n\nCaused by op 'MatMul', defined at:\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n    app.launch_new_instance()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n    app.start()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 477, in start\n    ioloop.IOLoop.instance().start()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tornado\\ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tornado\\stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 235, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 533, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2698, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2802, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2862, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-2-8b1e9efd65e5>\", line 17, in <module>\n    L1=tf.nn.tanh(tf.matmul(x,W1)+b1)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\ops\\math_ops.py\", line 2022, in matmul\n    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\ops\\gen_math_ops.py\", line 2799, in _mat_mul\n    name=name)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 3160, in create_op\n    op_def=op_def)\n  File \"D:\\soft\\+develop\\anaconda2_4.4.0\\envs\\py3\\lib\\site-packages\\tensorflow\\python\\framework\\ops.py\", line 1625, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nInternalError (see above for traceback): Blas GEMM launch failed : a.shape=(200, 784), b.shape=(784, 2000), m=200, n=2000, k=784\n\t [[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](_arg_Placeholder_0_0/_5, Variable/read)]]\n\t [[Node: add_7/_7 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_80_add_7\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\n"
     ]
    }
   ],
   "source": [
    "# 载入数据集\n",
    "mnist=input_data.read_data_sets(\"MNIST_data\",one_hot=True)\n",
    "\n",
    "# 每个批次的大小\n",
    "batch_size=200 # 每次放入的数据量\n",
    "# 计算有多少个批次\n",
    "n_batch=mnist.train.num_examples // batch_size\n",
    "\n",
    "# 定义两个placeholder\n",
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y=tf.placeholder(tf.float32,[None,10])\n",
    "keep_prob=tf.placeholder(tf.float32)\n",
    "\n",
    "# 创建简单的神经网络\n",
    "W1=tf.Variable(tf.truncated_normal([784,2000],stddev=0.1))\n",
    "b1=tf.Variable(tf.zeros([2000])+0.1)\n",
    "L1=tf.nn.tanh(tf.matmul(x,W1)+b1)\n",
    "L1_drop=tf.nn.dropout(L1,keep_prob)\n",
    "\n",
    "W2=tf.Variable(tf.truncated_normal([2000,2000],stddev=0.1))\n",
    "b2=tf.Variable(tf.zeros([2000])+0.1)\n",
    "L2=tf.nn.tanh(tf.matmul(L1_drop,W2)+b2)\n",
    "L2_drop=tf.nn.dropout(L2,keep_prob)\n",
    "\n",
    "W3=tf.Variable(tf.truncated_normal([2000,1000],stddev=0.1))\n",
    "b3=tf.Variable(tf.zeros([1000])+0.1)\n",
    "L3=tf.nn.tanh(tf.matmul(L2_drop,W3)+b3)\n",
    "L3_drop=tf.nn.dropout(L3,keep_prob)\n",
    "\n",
    "W4=tf.Variable(tf.truncated_normal([1000,10],stddev=0.1))\n",
    "b4=tf.Variable(tf.zeros([10])+0.1)\n",
    "prediction=tf.nn.softmax(tf.matmul(L3_drop,W4)+b4)\n",
    "\n",
    "# 二次代价函数\n",
    "# loss=tf.reduce_mean(tf.square(y-prediction))\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=prediction))\n",
    "\n",
    "# 使用梯度下降法\n",
    "train_step=tf.train.GradientDescentOptimizer(0.2).minimize(loss)\n",
    "\n",
    "# 记过存放在布尔型列表中\n",
    "correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(prediction,1)) # 最大值所在位置\n",
    "# 求准确率\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for epoch in range(50):\n",
    "        for batch in range(n_batch):\n",
    "            batch_xs,batch_ys=mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_step,feed_dict={x:batch_xs,y:batch_ys,keep_prob:1.0})\n",
    "        \n",
    "        test_acc = sess.run(accuracy,feed_dict={x:mnist.test.images,y:mnist.test.labels,keep_prob:1.0})\n",
    "        train_acc = sess.run(accuracy,feed_dict={x:mnist.train.images,y:mnist.train.labels,keep_prob:1.0})\n",
    "        print(\"Iter \" + str(epoch) + \",Testing Accuracy \" + str(test_acc) +\",Training Accuracy \" + str(train_acc))\n",
    "\n",
    "#         print(\"Iter\"+str(epoch)+\", Testing Accuracy:\"+str(acc))\n",
    "\n",
    "print('completed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "现在来试试随机失活会有什么效果："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting MNIST_data\\train-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\train-labels-idx1-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting MNIST_data\\t10k-labels-idx1-ubyte.gz\n",
      "Iter 0,Testing Accuracy 0.8557,Training Accuracy 0.846273\n",
      "Iter 1,Testing Accuracy 0.8885,Training Accuracy 0.881891\n",
      "Iter 2,Testing Accuracy 0.8994,Training Accuracy 0.893891\n",
      "Iter 3,Testing Accuracy 0.9088,Training Accuracy 0.904491\n",
      "Iter 4,Testing Accuracy 0.9137,Training Accuracy 0.910527\n",
      "Iter 5,Testing Accuracy 0.9192,Training Accuracy 0.913709\n",
      "Iter 6,Testing Accuracy 0.9205,Training Accuracy 0.918164\n",
      "Iter 7,Testing Accuracy 0.9233,Training Accuracy 0.921109\n",
      "Iter 8,Testing Accuracy 0.9273,Training Accuracy 0.924509\n",
      "Iter 9,Testing Accuracy 0.9284,Training Accuracy 0.927982\n",
      "Iter 10,Testing Accuracy 0.9312,Training Accuracy 0.9298\n",
      "Iter 11,Testing Accuracy 0.9317,Training Accuracy 0.9322\n",
      "Iter 12,Testing Accuracy 0.934,Training Accuracy 0.933691\n",
      "Iter 13,Testing Accuracy 0.933,Training Accuracy 0.935836\n",
      "Iter 14,Testing Accuracy 0.9352,Training Accuracy 0.938491\n",
      "Iter 15,Testing Accuracy 0.9352,Training Accuracy 0.938673\n",
      "Iter 16,Testing Accuracy 0.9365,Training Accuracy 0.939982\n",
      "Iter 17,Testing Accuracy 0.9382,Training Accuracy 0.942036\n",
      "Iter 18,Testing Accuracy 0.9414,Training Accuracy 0.942745\n",
      "Iter 19,Testing Accuracy 0.9376,Training Accuracy 0.944055\n",
      "Iter 20,Testing Accuracy 0.9415,Training Accuracy 0.945309\n",
      "Iter 21,Testing Accuracy 0.9433,Training Accuracy 0.9464\n",
      "Iter 22,Testing Accuracy 0.9432,Training Accuracy 0.946709\n",
      "Iter 23,Testing Accuracy 0.9445,Training Accuracy 0.949927\n",
      "Iter 24,Testing Accuracy 0.9461,Training Accuracy 0.949782\n",
      "Iter 25,Testing Accuracy 0.9479,Training Accuracy 0.951855\n",
      "Iter 26,Testing Accuracy 0.946,Training Accuracy 0.951327\n",
      "Iter 27,Testing Accuracy 0.9488,Training Accuracy 0.953273\n",
      "Iter 28,Testing Accuracy 0.9492,Training Accuracy 0.953873\n",
      "Iter 29,Testing Accuracy 0.9459,Training Accuracy 0.953818\n",
      "Iter 30,Testing Accuracy 0.9509,Training Accuracy 0.9538\n",
      "Iter 31,Testing Accuracy 0.9488,Training Accuracy 0.954545\n",
      "Iter 32,Testing Accuracy 0.9507,Training Accuracy 0.955455\n",
      "Iter 33,Testing Accuracy 0.9531,Training Accuracy 0.955818\n",
      "Iter 34,Testing Accuracy 0.9503,Training Accuracy 0.956855\n",
      "Iter 35,Testing Accuracy 0.9531,Training Accuracy 0.957782\n",
      "Iter 36,Testing Accuracy 0.9529,Training Accuracy 0.9578\n",
      "Iter 37,Testing Accuracy 0.9517,Training Accuracy 0.957927\n",
      "Iter 38,Testing Accuracy 0.9528,Training Accuracy 0.958709\n",
      "Iter 39,Testing Accuracy 0.9525,Training Accuracy 0.959491\n",
      "Iter 40,Testing Accuracy 0.9513,Training Accuracy 0.9608\n",
      "Iter 41,Testing Accuracy 0.9548,Training Accuracy 0.960364\n",
      "Iter 42,Testing Accuracy 0.9553,Training Accuracy 0.960218\n",
      "Iter 43,Testing Accuracy 0.9561,Training Accuracy 0.961818\n",
      "Iter 44,Testing Accuracy 0.955,Training Accuracy 0.961909\n",
      "Iter 45,Testing Accuracy 0.956,Training Accuracy 0.961927\n",
      "Iter 46,Testing Accuracy 0.9548,Training Accuracy 0.963564\n",
      "Iter 47,Testing Accuracy 0.9569,Training Accuracy 0.963145\n",
      "Iter 48,Testing Accuracy 0.9552,Training Accuracy 0.963509\n",
      "Iter 49,Testing Accuracy 0.957,Training Accuracy 0.963655\n",
      "completed\n"
     ]
    }
   ],
   "source": [
    "# 载入数据集\n",
    "mnist=input_data.read_data_sets(\"MNIST_data\",one_hot=True)\n",
    "\n",
    "# 每个批次的大小\n",
    "batch_size=200 # 每次放入的数据量\n",
    "# 计算有多少个批次\n",
    "n_batch=mnist.train.num_examples // batch_size\n",
    "\n",
    "# 定义两个placeholder\n",
    "x=tf.placeholder(tf.float32,[None,784])\n",
    "y=tf.placeholder(tf.float32,[None,10])\n",
    "keep_prob=tf.placeholder(tf.float32)\n",
    "\n",
    "# 创建简单的神经网络\n",
    "W1=tf.Variable(tf.truncated_normal([784,2000],stddev=0.1))\n",
    "b1=tf.Variable(tf.zeros([2000])+0.1)\n",
    "L1=tf.nn.tanh(tf.matmul(x,W1)+b1)\n",
    "L1_drop=tf.nn.dropout(L1,keep_prob)\n",
    "\n",
    "W2=tf.Variable(tf.truncated_normal([2000,2000],stddev=0.1))\n",
    "b2=tf.Variable(tf.zeros([2000])+0.1)\n",
    "L2=tf.nn.tanh(tf.matmul(L1_drop,W2)+b2)\n",
    "L2_drop=tf.nn.dropout(L2,keep_prob)\n",
    "\n",
    "W3=tf.Variable(tf.truncated_normal([2000,1000],stddev=0.1))\n",
    "b3=tf.Variable(tf.zeros([1000])+0.1)\n",
    "L3=tf.nn.tanh(tf.matmul(L2_drop,W3)+b3)\n",
    "L3_drop=tf.nn.dropout(L3,keep_prob)\n",
    "\n",
    "W4=tf.Variable(tf.truncated_normal([1000,10],stddev=0.1))\n",
    "b4=tf.Variable(tf.zeros([10])+0.1)\n",
    "prediction=tf.nn.softmax(tf.matmul(L3_drop,W4)+b4)\n",
    "\n",
    "# 二次代价函数\n",
    "# loss=tf.reduce_mean(tf.square(y-prediction))\n",
    "loss=tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y,logits=prediction))\n",
    "\n",
    "# 使用梯度下降法\n",
    "train_step=tf.train.GradientDescentOptimizer(0.2).minimize(loss)\n",
    "\n",
    "# 记过存放在布尔型列表中\n",
    "correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(prediction,1)) # 最大值所在位置\n",
    "# 求准确率\n",
    "accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    for epoch in range(50):\n",
    "        for batch in range(n_batch):\n",
    "            batch_xs,batch_ys=mnist.train.next_batch(batch_size)\n",
    "            sess.run(train_step,feed_dict={x:batch_xs,y:batch_ys,keep_prob:0.7})\n",
    "        \n",
    "        test_acc = sess.run(accuracy,feed_dict={x:mnist.test.images,y:mnist.test.labels,keep_prob:0.7})\n",
    "        train_acc = sess.run(accuracy,feed_dict={x:mnist.train.images,y:mnist.train.labels,keep_prob:0.7})\n",
    "        print(\"Iter \" + str(epoch) + \",Testing Accuracy \" + str(test_acc) +\",Training Accuracy \" + str(train_acc))\n",
    "\n",
    "#         print(\"Iter\"+str(epoch)+\", Testing Accuracy:\"+str(acc))\n",
    "\n",
    "print('completed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "收敛速度变慢，测试误差会一定程度的降低，但是训练误差和测试误差能尽可能接近，一定程度降低了过拟合，"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
